import requests
import re
import time
import math
import pymongo


MONGO_URI = 'localhost'
MONGO_DB = 'sinaCaiJing'
MONGO_COLLECTION = 'sina_caijing'


class SinaCaiJing(object):
    """爬取新浪财经基金数据中心信息
    """
    def __init__(self):
        self.base_url = "http://vip.stock.finance.sina.com.cn/fund_center/data/jsonp.php/IO.XSRV2.CallbackList['dxJ_DCLDxQnQYM8a']/NetValueReturn_Service.NetValueReturnOpen?page={}&num=40&sort=form_year&asc=0&ccode=&type2=0&type3="
        self.headers = {
            'Referer': 'http://vip.stock.finance.sina.com.cn/fund_center/index.html',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.170 Safari/537.36'
        }
        self.db = pymongo.MongoClient(MONGO_URI)[MONGO_DB]
        self.collection = MONGO_COLLECTION
    

    def get_content(self, url):
        respons = requests.get(url, headers=self.headers)
        return respons.content.decode('gbk')


    def get_raw_data(self, content):
        str_to_replace_1 = """//<script>location.href='http://sina.com.cn'; </script>
IO.XSRV2.CallbackList['dxJ_DCLDxQnQYM8a'](("""
        str_to_replace_2 = "))"
        raw_data = content.replace(str_to_replace_1, '').replace(str_to_replace_2, '')
        return raw_data
    

    def get_data_list(self, raw_data):
        # 要替换的字符串
        str_list_to_replace = ['symbol', 'sname', 'per_nav', 'total_nav', 'three_month', 'six_month', 'one_year', 'form_year','form_start', 'name', 'zmjgm', 'clrq', 'jjjl', 'dwjz', 'ljjz', 'jzrq', 'zjzfe', 'jjglr_code']
        # 数据列表字符串
        data_list = re.match(r"{total_num:(\d+),data:(\[.*?\])", raw_data, re.S).group(2)
        # 数据总量
        data_total_num = re.match(r"{total_num:(\d+),data:(\[.*?\])", raw_data, re.S).group(1)
        data_total_num = int(data_total_num)
        # 进行替换
        for i in str_list_to_replace:
            data_list = data_list.replace(i, '"' + i + '"')
        # 对重复替换的部分进行修正
        data_list = data_list.replace("s\"name\"", "sname")

        return eval(data_list), data_total_num
    

    def save_data(self, data_list):
        for data in data_list:
            self.db[self.collection].insert(data)
    

    def run_spider(self):
        page = 1
        while True:
            print('第', page, '页')
            print('='*20)
            time.sleep(0.1)
            url = self.base_url.format(str(page))
            content = self.get_content(url)
            raw_data = self.get_raw_data(content)
            data_list, data_total_num = self.get_data_list(raw_data)
            print(data_total_num)
            print(type(data_total_num))
            self.save_data(data_list)
            if page == math.ceil(data_total_num/40.0):
                break
            page += 1


def main():
    sina_cj = SinaCaiJing()
    sina_cj.run_spider()


if __name__ == '__main__':
    main()



